package com.bruce.tool.address.spider;

import com.bruce.tool.common.util.LogUtils;
import com.bruce.tool.rpc.http.core.Https;
import com.bruce.tool.rpc.http.handler.request.RequestType;
import com.bruce.tool.rpc.http.handler.response.ByteResponseHandler;
import com.google.common.base.Strings;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.RequestMethod;

import java.util.Objects;
import java.util.Random;

/**
 * 功能 :
 *
 * @author : Bruce(刘正航) 3:00 PM 2018/11/19
 */
@Slf4j
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public class HrefCrawler2 {

    private static Random random = new Random();

    /**根据url地址获取元数据**/
    public static byte[] fetchHtmlByUrl(String url, String referer) {
        int delay = random.nextInt(100);
        try {
            Thread.sleep(delay);
        } catch (InterruptedException e) {
            LogUtils.error(log,e);
            Thread.currentThread().interrupt();
        }

        LogUtils.info(log,"延迟:{}毫秒,请求地址:{}", Strings.padStart(String.valueOf(delay),4,'0'),url);
        // 根据不同的网站,设置不同的header,否则可能获取不到html内容
        byte[] first = Https.create().print(false)
                .url(url)
                .addHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
                .addHeader("Accept","ncoding: gzip, deflate")
                .addHeader("Accept","anguage: zh-CN,zh;q=0.9,en;q=0.8,id;q=0.7,ru;q=0.6,sq;q=0.5,zh-TW;q=0.4")
                .addHeader("Connection","keep-alive")
                .addHeader("Cookie","clickbids=55984; Hm_lvt_64ddd924d7a0b676880d576ade7db5a4=1583804104; Hm_lpvt_64ddd924d7a0b676880d576ade7db5a4="+System.currentTimeMillis()/1000)
                .addHeader("Host","www.bequgew.com")
                .addHeader("Upgrade","nsecure-Requests: 1")
                .addHeader("User","gent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36")
                .addHeader("Referer",referer)
                .retry(10,new Integer[]{200,500,1000,1500})
                .execute(RequestMethod.GET, RequestType.URL, new ByteResponseHandler());
        if(Objects.isNull(first) || first.length == 0 ){
            return new byte[0];
        }
        return first;
    }

}
